<!DOCTYPE html><html lang="zh-CN" data-theme="light"><head><meta charset="UTF-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width,initial-scale=1"><title>sketch算法 | 1006er的博客</title><meta name="keywords" content="sketch算法"><meta name="author" content="wrj"><meta name="copyright" content="wrj"><meta name="format-detection" content="telephone=no"><meta name="theme-color" content="#ffffff"><meta name="description" content="sketch">
<meta property="og:type" content="article">
<meta property="og:title" content="sketch算法">
<meta property="og:url" content="http://example.com/2021/11/16/sketch%E7%AE%97%E6%B3%95/index.html">
<meta property="og:site_name" content="1006er的博客">
<meta property="og:description" content="sketch">
<meta property="og:locale" content="zh_CN">
<meta property="og:image" content="https://gitee.com/wrj1006er/pic/raw/master/pic/01c10bca6b06d208a5b8e384c8edd9079fac5cb1.jpg@1320w_742h.webp">
<meta property="article:published_time" content="2021-11-16T01:39:14.000Z">
<meta property="article:modified_time" content="2021-12-02T02:13:51.982Z">
<meta property="article:author" content="wrj">
<meta property="article:tag" content="sketch算法">
<meta name="twitter:card" content="summary">
<meta name="twitter:image" content="https://gitee.com/wrj1006er/pic/raw/master/pic/01c10bca6b06d208a5b8e384c8edd9079fac5cb1.jpg@1320w_742h.webp"><link rel="shortcut icon" href="/img/favicon.png"><link rel="canonical" href="http://example.com/2021/11/16/sketch%E7%AE%97%E6%B3%95/"><link rel="preconnect" href="//cdn.jsdelivr.net"/><link rel="preconnect" href="//busuanzi.ibruce.info"/><link rel="stylesheet" href="/css/index.css"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@fortawesome/fontawesome-free/css/all.min.css" media="print" onload="this.media='all'"><script>const GLOBAL_CONFIG = { 
  root: '/',
  algolia: undefined,
  localSearch: undefined,
  translate: undefined,
  noticeOutdate: undefined,
  highlight: {"plugin":"highlighjs","highlightCopy":true,"highlightLang":true},
  copy: {
    success: '复制成功',
    error: '复制错误',
    noSupport: '浏览器不支持'
  },
  relativeDate: {
    homepage: false,
    post: false
  },
  runtime: '',
  date_suffix: {
    just: '刚刚',
    min: '分钟前',
    hour: '小时前',
    day: '天前',
    month: '个月前'
  },
  copyright: undefined,
  lightbox: 'fancybox',
  Snackbar: undefined,
  source: {
    jQuery: 'https://cdn.jsdelivr.net/npm/jquery@latest/dist/jquery.min.js',
    justifiedGallery: {
      js: 'https://cdn.jsdelivr.net/npm/justifiedGallery/dist/js/jquery.justifiedGallery.min.js',
      css: 'https://cdn.jsdelivr.net/npm/justifiedGallery/dist/css/justifiedGallery.min.css'
    },
    fancybox: {
      js: 'https://cdn.jsdelivr.net/npm/@fancyapps/fancybox@latest/dist/jquery.fancybox.min.js',
      css: 'https://cdn.jsdelivr.net/npm/@fancyapps/fancybox@latest/dist/jquery.fancybox.min.css'
    }
  },
  isPhotoFigcaption: false,
  islazyload: false,
  isanchor: false
}</script><script id="config-diff">var GLOBAL_CONFIG_SITE = { 
  isPost: true,
  isHome: false,
  isHighlightShrink: false,
  isToc: true,
  postUpdate: '2021-12-02 10:13:51'
}</script><noscript><style type="text/css">
  #nav {
    opacity: 1
  }
  .justified-gallery img {
    opacity: 1
  }

  #recent-posts time,
  #post-meta time {
    display: inline !important
  }
</style></noscript><link rel="stylesheet" href="APlayer.min.css"><div id="aplayer"></div><script src="https://cdn.jsdelivr.net/npm/meting@2/dist/Meting.min.js" async></script><script>(win=>{
    win.saveToLocal = {
      set: function setWithExpiry(key, value, ttl) {
        if (ttl === 0) return
        const now = new Date()
        const expiryDay = ttl * 86400000
        const item = {
          value: value,
          expiry: now.getTime() + expiryDay,
        }
        localStorage.setItem(key, JSON.stringify(item))
      },

      get: function getWithExpiry(key) {
        const itemStr = localStorage.getItem(key)

        if (!itemStr) {
          return undefined
        }
        const item = JSON.parse(itemStr)
        const now = new Date()

        if (now.getTime() > item.expiry) {
          localStorage.removeItem(key)
          return undefined
        }
        return item.value
      }
    }
  
    win.getScript = url => new Promise((resolve, reject) => {
      const script = document.createElement('script')
      script.src = url
      script.async = true
      script.onerror = reject
      script.onload = script.onreadystatechange = function() {
        const loadState = this.readyState
        if (loadState && loadState !== 'loaded' && loadState !== 'complete') return
        script.onload = script.onreadystatechange = null
        resolve()
      }
      document.head.appendChild(script)
    })
  
      win.activateDarkMode = function () {
        document.documentElement.setAttribute('data-theme', 'dark')
        if (document.querySelector('meta[name="theme-color"]') !== null) {
          document.querySelector('meta[name="theme-color"]').setAttribute('content', '#0d0d0d')
        }
      }
      win.activateLightMode = function () {
        document.documentElement.setAttribute('data-theme', 'light')
        if (document.querySelector('meta[name="theme-color"]') !== null) {
          document.querySelector('meta[name="theme-color"]').setAttribute('content', '#ffffff')
        }
      }
      const t = saveToLocal.get('theme')
    
          if (t === 'dark') activateDarkMode()
          else if (t === 'light') activateLightMode()
        
      const asideStatus = saveToLocal.get('aside-status')
      if (asideStatus !== undefined) {
        if (asideStatus === 'hide') {
          document.documentElement.classList.add('hide-aside')
        } else {
          document.documentElement.classList.remove('hide-aside')
        }
      }
    })(window)</script><script src="/live2d-widget/autoload.js"></script><meta name="generator" content="Hexo 5.3.0"><link rel="alternate" href="/atom.xml" title="1006er的博客" type="application/atom+xml">
</head><body><div id="sidebar"><div id="menu-mask"></div><div id="sidebar-menus"><div class="author-avatar"><img class="avatar-img" src="/img/avatar.png" onerror="onerror=null;src='/img/friend_404.gif'" alt="avatar"/></div><div class="site-data"><div class="data-item is-center"><div class="data-item-link"><a href="/archives/"><div class="headline">文章</div><div class="length-num">46</div></a></div></div><div class="data-item is-center"><div class="data-item-link"><a href="/tags/"><div class="headline">标签</div><div class="length-num">19</div></a></div></div><div class="data-item is-center"><div class="data-item-link"><a href="/categories/"><div class="headline">分类</div><div class="length-num">7</div></a></div></div></div><hr/><div class="menus_items"><div class="menus_item"><a class="site-page" href="/"><i class="fa-fw fas fa-home"></i><span> 首页</span></a></div><div class="menus_item"><a class="site-page" href="/archives/"><i class="fa-fw fas fa-archive"></i><span> 归档</span></a></div><div class="menus_item"><a class="site-page" href="/tags/"><i class="fa-fw fas fa-tags"></i><span> 标签</span></a></div><div class="menus_item"><a class="site-page" href="/categories/"><i class="fa-fw fas fa-folder-open"></i><span> 分类</span></a></div><div class="menus_item"><a class="site-page" href="javascript:void(0);"><i class="fa-fw fas fa-list"></i><span> 清单</span><i class="fas fa-chevron-down expand"></i></a><ul class="menus_item_child"><li><a class="site-page" href="/music/"><i class="fa-fw fas fa-music"></i><span> Music</span></a></li><li><a class="site-page" href="/movies/"><i class="fa-fw fas fa-video"></i><span> Movie</span></a></li></ul></div><div class="menus_item"><a class="site-page" href="/link/"><i class="fa-fw fas fa-link"></i><span> 友情链接</span></a></div><div class="menus_item"><a class="site-page" href="/about/"><i class="fa-fw fas fa-heart"></i><span> 关于</span></a></div></div></div></div><div class="post" id="body-wrap"><header class="post-bg" id="page-header" style="background-image: url('https://gitee.com/wrj1006er/pic/raw/master/pic/01c10bca6b06d208a5b8e384c8edd9079fac5cb1.jpg@1320w_742h.webp')"><nav id="nav"><span id="blog_name"><a id="site-name" href="/">1006er的博客</a></span><div id="menus"><div class="menus_items"><div class="menus_item"><a class="site-page" href="/"><i class="fa-fw fas fa-home"></i><span> 首页</span></a></div><div class="menus_item"><a class="site-page" href="/archives/"><i class="fa-fw fas fa-archive"></i><span> 归档</span></a></div><div class="menus_item"><a class="site-page" href="/tags/"><i class="fa-fw fas fa-tags"></i><span> 标签</span></a></div><div class="menus_item"><a class="site-page" href="/categories/"><i class="fa-fw fas fa-folder-open"></i><span> 分类</span></a></div><div class="menus_item"><a class="site-page" href="javascript:void(0);"><i class="fa-fw fas fa-list"></i><span> 清单</span><i class="fas fa-chevron-down expand"></i></a><ul class="menus_item_child"><li><a class="site-page" href="/music/"><i class="fa-fw fas fa-music"></i><span> Music</span></a></li><li><a class="site-page" href="/movies/"><i class="fa-fw fas fa-video"></i><span> Movie</span></a></li></ul></div><div class="menus_item"><a class="site-page" href="/link/"><i class="fa-fw fas fa-link"></i><span> 友情链接</span></a></div><div class="menus_item"><a class="site-page" href="/about/"><i class="fa-fw fas fa-heart"></i><span> 关于</span></a></div></div><div id="toggle-menu"><a class="site-page"><i class="fas fa-bars fa-fw"></i></a></div></div></nav><div id="post-info"><h1 class="post-title">sketch算法</h1><div id="post-meta"><div class="meta-firstline"><span class="post-meta-date"><i class="far fa-calendar-alt fa-fw post-meta-icon"></i><span class="post-meta-label">发表于</span><time class="post-meta-date-created" datetime="2021-11-16T01:39:14.000Z" title="发表于 2021-11-16 09:39:14">2021-11-16</time><span class="post-meta-separator">|</span><i class="fas fa-history fa-fw post-meta-icon"></i><span class="post-meta-label">更新于</span><time class="post-meta-date-updated" datetime="2021-12-02T02:13:51.982Z" title="更新于 2021-12-02 10:13:51">2021-12-02</time></span><span class="post-meta-categories"><span class="post-meta-separator">|</span><i class="fas fa-inbox fa-fw post-meta-icon"></i><a class="post-meta-categories" href="/categories/sketch/">sketch</a></span></div><div class="meta-secondline"><span class="post-meta-separator">|</span><span class="post-meta-pv-cv"><i class="far fa-eye fa-fw post-meta-icon"></i><span class="post-meta-label">阅读量:</span><span id="busuanzi_value_page_pv"></span></span></div></div></div></header><main class="layout" id="content-inner"><div id="post"><article class="post-content" id="article-container"><h1 id="sketch算法总结"><a href="#sketch算法总结" class="headerlink" title="sketch算法总结"></a>sketch算法总结</h1><h2 id="sketch算法的目的"><a href="#sketch算法的目的" class="headerlink" title="sketch算法的目的"></a>sketch算法的目的</h2><p>sketch统计网络数据流中某个元素出现的频率，反应数据流的特征。并不实际的存储数据流中的元素，只存储他们的计数。<br>如果一个数据流{A1,A2,A3……Am}，Ai∈{D1,D2,D3,D4….Dn}其中m为数据流的大小，。我们可以定义每个元素出现的频率为f = {f1,f2,f3…fn},fi就是Di元素出现的次数.<br>数据流里面的数据的种类不确定，采用一种固定大小的数据结构进行存储</p>
<h2 id="Count-Min-CM-Sketch"><a href="#Count-Min-CM-Sketch" class="headerlink" title="Count-Min (CM) Sketch"></a>Count-Min (CM) Sketch</h2><p><img src="https://gitee.com/wrj1006er/pic/raw/master/pic/27AE825AD09DE23F62FF5FBA031198BB.png" alt="RUNOOB 图标"></p>
<p>1.单个哈希函数，一维<br>两个不同的数据可经过哈希函数处理后可能冲突，很容易出现重复的，并且概率很高。</p>
<p>2.Count-Min (CM) Sketch就是调用上述方法多次取最小值。</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br></pre></td><td class="code"><pre><span class="line"># 假设d&#x3D;3, 单个哈希表的宽度为w</span><br><span class="line">哈希表1 &#x3D; [0]*w</span><br><span class="line">哈希表2 &#x3D; [0]*w</span><br><span class="line">哈希表3 &#x3D; [0]*w</span><br><span class="line">def CountMin(数据包):</span><br><span class="line">	标识符 &#x3D; 数据包.流标识符</span><br><span class="line">	索引1 &#x3D; h1(标识符)%w</span><br><span class="line">	索引2 &#x3D; h2(标识符)%w</span><br><span class="line">	索引3 &#x3D; h3(标识符)%w</span><br><span class="line">	哈希表1[索引1] +&#x3D; 1</span><br><span class="line">	哈希表2[索引2] +&#x3D; 1</span><br><span class="line">	哈希表3[索引3] +&#x3D; 1</span><br><span class="line"></span><br><span class="line">#查询</span><br><span class="line">def 查询(f):</span><br><span class="line">	标识符 &#x3D; f.流标识符</span><br><span class="line">	索引1 &#x3D; h1(标识符)%w</span><br><span class="line">	索引2 &#x3D; h2(标识符)%w</span><br><span class="line">	索引3 &#x3D; h3(标识符)%w</span><br><span class="line">	长度 &#x3D; min(哈希表1[索引1], 哈希表2[索引2], 哈希表3[索引3])</span><br><span class="line">	return 长度</span><br></pre></td></tr></table></figure>
<p><img src="https://gitee.com/wrj1006er/pic/raw/master/pic/34a6f88f888cb12259dfbd4ceb2c78a0.png" alt="RUNOOB 图标"></p>
<p>对于这个要查询的数据，数值最小的代表数据冲突是最小的。</p>
<h2 id="Conservative-Update-CU-Sketch"><a href="#Conservative-Update-CU-Sketch" class="headerlink" title="Conservative-Update (CU) Sketch"></a>Conservative-Update (CU) Sketch</h2><p>相对于Count-Min (CM) Sketch的改进</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br></pre></td><td class="code"><pre><span class="line"># 假设d&#x3D;3, 单个哈希表的宽度为w</span><br><span class="line">哈希表1 &#x3D; [0]*w</span><br><span class="line">哈希表2 &#x3D; [0]*w</span><br><span class="line">哈希表3 &#x3D; [0]*w</span><br><span class="line">def ConservativeUpdate(数据包):</span><br><span class="line">	标识符 &#x3D; 数据包.流标识符</span><br><span class="line">	索引1 &#x3D; h1(标识符)%w</span><br><span class="line">	索引2 &#x3D; h2(标识符)%w</span><br><span class="line">	索引3 &#x3D; h3(标识符)%w</span><br><span class="line">	最小计数值 &#x3D; min(哈希表1[索引1], 哈希表2[索引2], 哈希表3[索引3]) + 1</span><br><span class="line">	哈希表1[索引1] &#x3D; max(哈希表1[索引1], 最小计数值)</span><br><span class="line">	哈希表2[索引2] &#x3D; max(哈希表2[索引2], 最小计数值)</span><br><span class="line">	哈希表3[索引3] &#x3D; max(哈希表3[索引3], 最小计数值)</span><br><span class="line"></span><br><span class="line">#查询</span><br><span class="line">def 查询(f):</span><br><span class="line">	标识符 &#x3D; f.流标识符</span><br><span class="line">	索引1 &#x3D; h1(标识符)%w</span><br><span class="line">	索引2 &#x3D; h2(标识符)%w</span><br><span class="line">	索引3 &#x3D; h3(标识符)%w</span><br><span class="line">	长度 &#x3D; min(哈希表1[索引1], 哈希表2[索引2], 哈希表3[索引3])</span><br><span class="line">	return 长度</span><br></pre></td></tr></table></figure>
<p>对Count-Min (CM) Sketch的改进在于哈希表更新的地方：只对最小值的那个进行更新。因为如果在不同的哈希表中出现不同的值，代表较大的几个都出现了冲突。所以只要更新最小的就可以了。</p>
<h2 id="Count-Mean-Min-Sketch算法"><a href="#Count-Mean-Min-Sketch算法" class="headerlink" title="Count-Mean-Min Sketch算法"></a>Count-Mean-Min Sketch算法</h2><h3 id="插入"><a href="#插入" class="headerlink" title="插入"></a>插入</h3><p>插入数据与cm算法一样</p>
<h3 id="读取"><a href="#读取" class="headerlink" title="读取"></a>读取</h3><p>在读取的时候增加了一步噪声的的计算，在最终读取到数据后减去这部分的噪音。</p>
<h3 id="算法流程"><a href="#算法流程" class="headerlink" title="算法流程"></a>算法流程</h3><p>CountMinSketch算法的流程：</p>
<ul>
<li>来了一个查询，按照 Count-Min Sketch的正常流程，取出它的d个sketch</li>
<li>对于每个hash函数，估算出一个噪音，噪音等于该行所有整数(除了被查询的这个元素)的平均值</li>
<li>用该行的sketch 减去该行的噪音，作为真正的sketch</li>
<li>返回d个sketch的中位数<br><img src="https://gitee.com/wrj1006er/pic/raw/master/pic/QQ%E6%88%AA%E5%9B%BE20211125170625.jpg" alt="RUNOOB 图标"><h3 id="总结"><a href="#总结" class="headerlink" title="总结"></a>总结</h3>Count-Mean-Min Sketch算法能够显著的改善在长尾数据上的精确度。</li>
</ul>
<h2 id="Count-Min-Log-Sketch算法"><a href="#Count-Min-Log-Sketch算法" class="headerlink" title="Count-Min-Log Sketch算法"></a>Count-Min-Log Sketch算法</h2><p>对于传统的sketch我们计算统计的都是可能出现的最大的值。而大多数counter由于记录的是低频项，因此并不需要那么大的空间，这就造成了空间浪费。因此CML每次只以x^(-c)的概率增加counter的计数，其中c为对当前需要插入元素的估计值，x为大于1的log base，且增加计数时采用了CU策略。对应的插入和查询算法修改如下：</p>
<h3 id="插入-1"><a href="#插入-1" class="headerlink" title="插入"></a>插入</h3><p><img src="https://gitee.com/wrj1006er/pic/raw/master/pic/20180326212812623.png" alt="RUNOOB 图标"></p>
<h3 id="查询"><a href="#查询" class="headerlink" title="查询"></a>查询</h3><p><img src="https://gitee.com/wrj1006er/pic/raw/master/pic/20180326212908587.png" alt="RUNOOB 图标"></p>
<h2 id="Count-Sketch算法"><a href="#Count-Sketch算法" class="headerlink" title="Count Sketch算法"></a>Count Sketch算法</h2><p>引入了随机函数，生成1或者-1.</p>
<h3 id="Sketch"><a href="#Sketch" class="headerlink" title="Sketch"></a>Sketch</h3><figure class="highlight c++"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">//Counts[]:计数数组，大小为k</span></span><br><span class="line"><span class="comment">//hash function h:[n]-&gt;[k]</span></span><br><span class="line"><span class="comment">//hash function h:[n]-&gt;&#123;1,-1&#125;</span></span><br><span class="line"><span class="function"><span class="keyword">void</span> <span class="title">Process</span><span class="params">(<span class="built_in">vector</span>&lt;<span class="keyword">int</span>&gt;vec)</span></span>&#123;<span class="comment">//处理整个数据流</span></span><br><span class="line">	<span class="keyword">for</span>(<span class="keyword">int</span> i=<span class="number">0</span>;i&lt;vec.size();i++)&#123;</span><br><span class="line">		C[h(vec[i])]=C[h(vec[i])]+g(j);</span><br><span class="line">	&#125;</span><br><span class="line">&#125;</span><br><span class="line"><span class="function"><span class="keyword">int</span> <span class="title">query</span><span class="params">(<span class="keyword">int</span> a)</span></span>&#123;<span class="comment">//查询a元素出现次数</span></span><br><span class="line">	<span class="keyword">return</span> g(a)*C[h(a)];</span><br><span class="line">&#125;</span><br></pre></td></tr></table></figure>
<ul>
<li>对于一个特定的数据来说，可能对于哈希表的计数是+1或者-1，但是我们在查询的时候，对于如果此数的计数方法是+1，取它在哈希表中的值，如果计数方法是-1的则取反。</li>
<li>对于其他冲突的数值来说，因为每个数据的技术方法是+1，还是-1是随机选择的，所以在数据量大的时候，可以相互抵消，这样就使得查询结果相对正确。</li>
</ul>
<h3 id="Count-Sketch"><a href="#Count-Sketch" class="headerlink" title="Count Sketch"></a>Count Sketch</h3><p>调用Sketch方法K次，如何取中位数。</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br></pre></td><td class="code"><pre><span class="line"># 假设d&#x3D;3, 单个哈希表的宽度为w</span><br><span class="line">哈希表1 &#x3D; [0]*w</span><br><span class="line">哈希表2 &#x3D; [0]*w</span><br><span class="line">哈希表3 &#x3D; [0]*w</span><br><span class="line">def CountSketch(数据包):</span><br><span class="line">	标识符 &#x3D; 数据包.流标识符</span><br><span class="line">	索引1 &#x3D; h1(标识符)%w</span><br><span class="line">	索引2 &#x3D; h2(标识符)%w</span><br><span class="line">	索引3 &#x3D; h3(标识符)%w</span><br><span class="line">	哈希表1[索引1] +&#x3D; s1(标识符)</span><br><span class="line">	哈希表2[索引2] +&#x3D; s2(标识符)</span><br><span class="line">	哈希表3[索引3] +&#x3D; s3(标识符)</span><br><span class="line"></span><br><span class="line">def 查询(f):</span><br><span class="line">	标识符 &#x3D; f.流标识符</span><br><span class="line">	索引1 &#x3D; h1(标识符)%w</span><br><span class="line">	索引2 &#x3D; h2(标识符)%w</span><br><span class="line">	索引3 &#x3D; h3(标识符)%w</span><br><span class="line">	长度 &#x3D; 中位数(哈希表1[索引1]*s1(标识符), 哈希表2[索引2]*s2(标识符), 哈希表3[索引3]*s3(标识符))</span><br><span class="line">	return 长度</span><br></pre></td></tr></table></figure>
<h2 id="Augmented-Sketch"><a href="#Augmented-Sketch" class="headerlink" title="Augmented  Sketch"></a>Augmented  Sketch</h2><p>引入了过滤器，过滤器开源与sketch进行元素交换(exchange)。过滤器中的每个元素都有new_count和old_count两个值，new_count保持对数据项的过高估计(如同原先的sketch)，而new_count和old_count间的差值表示在某段时间内该数据项的累计值，这里使用的sketch为CM。</p>
<h3 id="插入操作"><a href="#插入操作" class="headerlink" title="插入操作"></a>插入操作</h3><p>对二元组(k,u)进行插入操作，k为元素，u为次数：<br>1、如果过滤器中有k，直接对其new_count加u，而old_count不变</p>
<p>2、如果没有且过滤器未满，把k加入并设其new_count = u, old_count =0</p>
<p>3、如果没有且过滤器已满，先把k加入sketch，若此时对k的估计值大于过滤器中的最小值，则将该最小值散列到sketch中共 (new_count - old_count)次，并把k加入过滤器，new_count和old_count都设为原先k的估计值<br><img src="https://gitee.com/wrj1006er/pic/raw/master/pic/20180326215403981.png" alt="RUNOOB 图标"></p>
<h3 id="读取操作"><a href="#读取操作" class="headerlink" title="读取操作"></a>读取操作</h3><p>先在过滤器中读，然后再在sketch中查找</p>
<h3 id="总结-1"><a href="#总结-1" class="headerlink" title="总结"></a>总结</h3><p>优点就是节省了大量的hash查找，且因为高频项大部分时间都存在过滤器中，而过滤器的计数是完全精确的，因此可以大大提高对高频项的估计情况。</p>
</article><div class="post-copyright"><div class="post-copyright__author"><span class="post-copyright-meta">文章作者: </span><span class="post-copyright-info"><a href="mailto:undefined">wrj</a></span></div><div class="post-copyright__type"><span class="post-copyright-meta">文章链接: </span><span class="post-copyright-info"><a href="http://example.com/2021/11/16/sketch%E7%AE%97%E6%B3%95/">http://example.com/2021/11/16/sketch%E7%AE%97%E6%B3%95/</a></span></div><div class="post-copyright__notice"><span class="post-copyright-meta">版权声明: </span><span class="post-copyright-info">本博客所有文章除特别声明外，均采用 <a href="https://creativecommons.org/licenses/by-nc-sa/4.0/" target="_blank">CC BY-NC-SA 4.0</a> 许可协议。转载请注明来自 <a href="http://example.com" target="_blank">1006er的博客</a>！</span></div></div><div class="tag_share"><div class="post-meta__tag-list"><a class="post-meta__tags" href="/tags/sketch%E7%AE%97%E6%B3%95/">sketch算法</a></div><div class="post_share"><div class="social-share" data-image="https://gitee.com/wrj1006er/pic/raw/master/pic/01c10bca6b06d208a5b8e384c8edd9079fac5cb1.jpg@1320w_742h.webp" data-sites="facebook,twitter,wechat,weibo,qq"></div><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/social-share.js/dist/css/share.min.css" media="print" onload="this.media='all'"><script src="https://cdn.jsdelivr.net/npm/social-share.js/dist/js/social-share.min.js" defer></script></div></div><nav class="pagination-post" id="pagination"><div class="prev-post pull-left"><a href="/2021/11/26/SCAFFOLD-Stochastic-Controlled-Averaging-for-Federated-Learning/"><img class="prev-cover" src="https://cdn.jsdelivr.net/npm/butterfly-extsrc@1/img/default.jpg" onerror="onerror=null;src='/img/404.jpg'" alt="cover of previous post"><div class="pagination-info"><div class="label">上一篇</div><div class="prev_info">SCAFFOLD: Stochastic Controlled Averaging for Federated Learning</div></div></a></div><div class="next-post pull-right"><a href="/2021/11/08/FetchSGD-Communication-Efficient-Federated-Learning-with-Sketching/"><img class="next-cover" src="https://gitee.com/wrj1006er/pic/raw/master/pic/09d41287162d99b11293fb5e06ad965bb0247a1c.jpg@942w_531h_progressive.webp" onerror="onerror=null;src='/img/404.jpg'" alt="cover of next post"><div class="pagination-info"><div class="label">下一篇</div><div class="next_info">FetchSGD: Communication-Efficient Federated Learning with Sketching</div></div></a></div></nav></div><div class="aside-content" id="aside-content"><div class="card-widget card-info"><div class="card-info-avatar is-center"><img class="avatar-img" src="/img/avatar.png" onerror="this.onerror=null;this.src='/img/friend_404.gif'" alt="avatar"/><div class="author-info__name">wrj</div><div class="author-info__description"></div></div><div class="card-info-data"><div class="card-info-data-item is-center"><a href="/archives/"><div class="headline">文章</div><div class="length-num">46</div></a></div><div class="card-info-data-item is-center"><a href="/tags/"><div class="headline">标签</div><div class="length-num">19</div></a></div><div class="card-info-data-item is-center"><a href="/categories/"><div class="headline">分类</div><div class="length-num">7</div></a></div></div><a class="button--animated" id="card-info-btn" target="_blank" rel="noopener" href="https://github.com/xxxxxx"><i class="fab fa-github"></i><span>Follow Me</span></a></div><div class="card-widget card-announcement"><div class="item-headline"><i class="fas fa-bullhorn card-announcement-animation"></i><span>公告</span></div><div class="announcement_content">This is my Blog</div></div><div class="sticky_layout"><div class="card-widget" id="card-toc"><div class="item-headline"><i class="fas fa-stream"></i><span>目录</span></div><div class="toc-content"><ol class="toc"><li class="toc-item toc-level-1"><a class="toc-link" href="#sketch%E7%AE%97%E6%B3%95%E6%80%BB%E7%BB%93"><span class="toc-number">1.</span> <span class="toc-text">sketch算法总结</span></a><ol class="toc-child"><li class="toc-item toc-level-2"><a class="toc-link" href="#sketch%E7%AE%97%E6%B3%95%E7%9A%84%E7%9B%AE%E7%9A%84"><span class="toc-number">1.1.</span> <span class="toc-text">sketch算法的目的</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#Count-Min-CM-Sketch"><span class="toc-number">1.2.</span> <span class="toc-text">Count-Min (CM) Sketch</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#Conservative-Update-CU-Sketch"><span class="toc-number">1.3.</span> <span class="toc-text">Conservative-Update (CU) Sketch</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#Count-Mean-Min-Sketch%E7%AE%97%E6%B3%95"><span class="toc-number">1.4.</span> <span class="toc-text">Count-Mean-Min Sketch算法</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#%E6%8F%92%E5%85%A5"><span class="toc-number">1.4.1.</span> <span class="toc-text">插入</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E8%AF%BB%E5%8F%96"><span class="toc-number">1.4.2.</span> <span class="toc-text">读取</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E7%AE%97%E6%B3%95%E6%B5%81%E7%A8%8B"><span class="toc-number">1.4.3.</span> <span class="toc-text">算法流程</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E6%80%BB%E7%BB%93"><span class="toc-number">1.4.4.</span> <span class="toc-text">总结</span></a></li></ol></li><li class="toc-item toc-level-2"><a class="toc-link" href="#Count-Min-Log-Sketch%E7%AE%97%E6%B3%95"><span class="toc-number">1.5.</span> <span class="toc-text">Count-Min-Log Sketch算法</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#%E6%8F%92%E5%85%A5-1"><span class="toc-number">1.5.1.</span> <span class="toc-text">插入</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E6%9F%A5%E8%AF%A2"><span class="toc-number">1.5.2.</span> <span class="toc-text">查询</span></a></li></ol></li><li class="toc-item toc-level-2"><a class="toc-link" href="#Count-Sketch%E7%AE%97%E6%B3%95"><span class="toc-number">1.6.</span> <span class="toc-text">Count Sketch算法</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#Sketch"><span class="toc-number">1.6.1.</span> <span class="toc-text">Sketch</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#Count-Sketch"><span class="toc-number">1.6.2.</span> <span class="toc-text">Count Sketch</span></a></li></ol></li><li class="toc-item toc-level-2"><a class="toc-link" href="#Augmented-Sketch"><span class="toc-number">1.7.</span> <span class="toc-text">Augmented  Sketch</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#%E6%8F%92%E5%85%A5%E6%93%8D%E4%BD%9C"><span class="toc-number">1.7.1.</span> <span class="toc-text">插入操作</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E8%AF%BB%E5%8F%96%E6%93%8D%E4%BD%9C"><span class="toc-number">1.7.2.</span> <span class="toc-text">读取操作</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E6%80%BB%E7%BB%93-1"><span class="toc-number">1.7.3.</span> <span class="toc-text">总结</span></a></li></ol></li></ol></li></ol></div></div><div class="card-widget card-recent-post"><div class="item-headline"><i class="fas fa-history"></i><span>最新文章</span></div><div class="aside-list"><div class="aside-list-item"><a class="thumbnail" href="/2022/01/19/redis-cluster/" title="redis-cluster"><img src="https://cdn.jsdelivr.net/npm/butterfly-extsrc@1/img/default.jpg" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="redis-cluster"/></a><div class="content"><a class="title" href="/2022/01/19/redis-cluster/" title="redis-cluster">redis-cluster</a><time datetime="2022-01-19T13:23:30.000Z" title="发表于 2022-01-19 21:23:30">2022-01-19</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/2022/01/17/redis%E9%9B%86%E7%BE%A4/" title="redis集群"><img src="https://cdn.jsdelivr.net/npm/butterfly-extsrc@1/img/default.jpg" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="redis集群"/></a><div class="content"><a class="title" href="/2022/01/17/redis%E9%9B%86%E7%BE%A4/" title="redis集群">redis集群</a><time datetime="2022-01-17T07:08:35.000Z" title="发表于 2022-01-17 15:08:35">2022-01-17</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/2021/12/31/%E5%B7%A5%E4%B8%9A%E4%BA%92%E8%81%94%E7%BD%91%E7%AB%AF%E8%BE%B9%E4%BA%91%E5%8D%8F%E5%90%8C%E6%95%B0%E6%8D%AE%E5%90%8C%E6%AD%A5%E6%96%B9%E6%A1%88%E8%AE%BE%E8%AE%A1%E4%B8%8E%E5%AE%9E%E7%8E%B0%E8%AE%BA%E6%96%87%E7%AC%94%E8%AE%B0/" title="工业互联网端边云协同数据同步方案设计与实现论文笔记"><img src="https://cdn.jsdelivr.net/npm/butterfly-extsrc@1/img/default.jpg" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="工业互联网端边云协同数据同步方案设计与实现论文笔记"/></a><div class="content"><a class="title" href="/2021/12/31/%E5%B7%A5%E4%B8%9A%E4%BA%92%E8%81%94%E7%BD%91%E7%AB%AF%E8%BE%B9%E4%BA%91%E5%8D%8F%E5%90%8C%E6%95%B0%E6%8D%AE%E5%90%8C%E6%AD%A5%E6%96%B9%E6%A1%88%E8%AE%BE%E8%AE%A1%E4%B8%8E%E5%AE%9E%E7%8E%B0%E8%AE%BA%E6%96%87%E7%AC%94%E8%AE%B0/" title="工业互联网端边云协同数据同步方案设计与实现论文笔记">工业互联网端边云协同数据同步方案设计与实现论文笔记</a><time datetime="2021-12-31T08:47:17.000Z" title="发表于 2021-12-31 16:47:17">2021-12-31</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/2021/12/31/RSYNC/" title="RSYNC"><img src="https://cdn.jsdelivr.net/npm/butterfly-extsrc@1/img/default.jpg" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="RSYNC"/></a><div class="content"><a class="title" href="/2021/12/31/RSYNC/" title="RSYNC">RSYNC</a><time datetime="2021-12-31T07:36:41.000Z" title="发表于 2021-12-31 15:36:41">2021-12-31</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/2021/12/17/%E7%AE%97%E5%8A%9B%E8%B5%84%E6%BA%90%E4%B8%8E%E7%8A%B6%E6%80%81%E6%9C%8D%E5%8A%A1%E4%BF%A1%E6%81%AF%E7%9A%84%E5%8E%8B%E7%BC%A9%E7%AE%97%E6%B3%95%E8%B0%83%E7%A0%94/" title="算力资源与状态服务信息的压缩算法调研"><img src="https://cdn.jsdelivr.net/npm/butterfly-extsrc@1/img/default.jpg" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="算力资源与状态服务信息的压缩算法调研"/></a><div class="content"><a class="title" href="/2021/12/17/%E7%AE%97%E5%8A%9B%E8%B5%84%E6%BA%90%E4%B8%8E%E7%8A%B6%E6%80%81%E6%9C%8D%E5%8A%A1%E4%BF%A1%E6%81%AF%E7%9A%84%E5%8E%8B%E7%BC%A9%E7%AE%97%E6%B3%95%E8%B0%83%E7%A0%94/" title="算力资源与状态服务信息的压缩算法调研">算力资源与状态服务信息的压缩算法调研</a><time datetime="2021-12-17T08:34:12.000Z" title="发表于 2021-12-17 16:34:12">2021-12-17</time></div></div></div></div></div></div></main><footer id="footer"><div id="footer-wrap"><div class="copyright">&copy;2020 - 2022 By wrj</div><div class="framework-info"><span>框架 </span><a target="_blank" rel="noopener" href="https://hexo.io">Hexo</a><span class="footer-separator">|</span><span>主题 </span><a target="_blank" rel="noopener" href="https://github.com/jerryc127/hexo-theme-butterfly">Butterfly</a></div></div></footer></div><div id="rightside"><div id="rightside-config-hide"><button id="readmode" type="button" title="阅读模式"><i class="fas fa-book-open"></i></button><button id="darkmode" type="button" title="浅色和深色模式转换"><i class="fas fa-adjust"></i></button><button id="hide-aside-btn" type="button" title="单栏和双栏切换"><i class="fas fa-arrows-alt-h"></i></button></div><div id="rightside-config-show"><button id="rightside_config" type="button" title="设置"><i class="fas fa-cog fa-spin"></i></button><button class="close" id="mobile-toc-button" type="button" title="目录"><i class="fas fa-list-ul"></i></button><button id="go-up" type="button" title="回到顶部"><i class="fas fa-arrow-up"></i></button></div></div><div><script src="/js/utils.js"></script><script src="/js/main.js"></script><div class="js-pjax"></div><script async data-pjax src="//busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js"></script></div></body></html>